
<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
      
      <link rel="icon" href="../../../../static/images/favicon.png">
      <meta name="generator" content="mkdocs-1.3.0, mkdocs-material-8.2.8">
    
    
      
        <title>基于 Apache Hadoop 大数据平台生产部署（原汁原味） - WL4G DOCS</title>
      
    
    
      <link rel="stylesheet" href="../../../../assets/stylesheets/main.644de097.min.css">
      
        
        <link rel="stylesheet" href="../../../../assets/stylesheets/palette.e6a45f82.min.css">
        
      
    
    
    
      
        
        
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    
      <link rel="stylesheet" href="../../../../static/css/util.css">
    
    <script>__md_scope=new URL("../../../..",location),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      

    
    
  </head>
  
  
    
    
      
    
    
    
    
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent="">
  
    
    
      <script>var palette=__md_get("__palette");if(palette&&"object"==typeof palette.color)for(var key of Object.keys(palette.color))document.body.setAttribute("data-md-color-"+key,palette.color[key])</script>
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#apache-hadoop" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
      <div data-md-component="outdated" hidden>
        <aside class="md-banner md-banner--warning">
          
        </aside>
      </div>
    
    
      

<header class="md-header" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../../../.." title="WL4G DOCS" class="md-header__button md-logo" aria-label="WL4G DOCS" data-md-component="logo">
      
  <img src="../../../../static/images/mylogo.jpeg" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            WL4G DOCS
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              基于 Apache Hadoop 大数据平台生产部署（原汁原味）
            
          </span>
        </div>
      </div>
    </div>
    
      <form class="md-header__option" data-md-component="palette">
        
          
          
          <input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="" data-md-color-accent=""  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_1">
          
            <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_2" hidden>
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 6H7c-3.31 0-6 2.69-6 6s2.69 6 6 6h10c3.31 0 6-2.69 6-6s-2.69-6-6-6zm0 10H7c-2.21 0-4-1.79-4-4s1.79-4 4-4h10c2.21 0 4 1.79 4 4s-1.79 4-4 4zM7 9c-1.66 0-3 1.34-3 3s1.34 3 3 3 3-1.34 3-3-1.34-3-3-3z"/></svg>
            </label>
          
        
          
          
          <input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="" data-md-color-accent=""  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_2">
          
            <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
              <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 7H7a5 5 0 0 0-5 5 5 5 0 0 0 5 5h10a5 5 0 0 0 5-5 5 5 0 0 0-5-5m0 8a3 3 0 0 1-3-3 3 3 0 0 1 3-3 3 3 0 0 1 3 3 3 3 0 0 1-3 3z"/></svg>
            </label>
          
        
      </form>
    
    
      <div class="md-header__option">
        <div class="md-select">
          
          <button class="md-header__button md-icon" aria-label="Select language">
            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m12.87 15.07-2.54-2.51.03-.03A17.52 17.52 0 0 0 14.07 6H17V4h-7V2H8v2H1v2h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04M18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12m-2.62 7 1.62-4.33L19.12 17h-3.24z"/></svg>
          </button>
          <div class="md-select__inner">
            <ul class="md-select__list">
              
                <li class="md-select__item">
                  <a href="/en/" hreflang="en" class="md-select__link">
                    English
                  </a>
                </li>
                
                <li class="md-select__item">
                  <a href="/zh/" hreflang="zh" class="md-select__link">
                    简体中文
                  </a>
                </li>
                
            </ul>
          </div>
        </div>
      </div>
    
    
      <label class="md-header__button md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5z"/></svg>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" aria-label="Clear" tabindex="-1">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41z"/></svg>
        </button>
      </nav>
      
        <div class="md-search__suggest" data-md-component="search-suggest"></div>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
  </nav>
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
        
          
            
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
  <div class="md-tabs__inner md-grid">
    <ul class="md-tabs__list">
      
        
  
  


  
  
  
    <li class="md-tabs__item">
      <a href="../../../.." class="md-tabs__link">
        Getting Started
      </a>
    </li>
  

      
    </ul>
  </div>
</nav>
          
        
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    

  


  

<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../../../.." title="WL4G DOCS" class="md-nav__button md-logo" aria-label="WL4G DOCS" data-md-component="logo">
      
  <img src="../../../../static/images/mylogo.jpeg" alt="logo">

    </a>
    WL4G DOCS
  </label>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
      

  
  
  
    
    <li class="md-nav__item md-nav__item--nested">
      
      
        <input class="md-nav__toggle md-toggle" data-md-toggle="__nav_1" data-md-state="indeterminate" type="checkbox" id="__nav_1" checked>
      
      
      
      
        <label class="md-nav__link" for="__nav_1">
          Getting Started
          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <nav class="md-nav" aria-label="Getting Started" data-md-level="1">
        <label class="md-nav__title" for="__nav_1">
          <span class="md-nav__icon md-icon"></span>
          Getting Started
        </label>
        <ul class="md-nav__list" data-md-scrollfix>
          
            
              
  
  
  
    <li class="md-nav__item">
      <a href="../../../.." class="md-nav__link">
        Introduction
      </a>
    </li>
  

            
          
            
              
  
  
  
    <li class="md-nav__item">
      <a href="../../../../ABOUT_CN/" class="md-nav__link">
        About
      </a>
    </li>
  

            
          
        </ul>
      </nav>
    </li>
  

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
          
          <div class="md-content" data-md-component="content">
            <article class="md-content__inner md-typeset">
              
                


<h1 id="apache-hadoop">基于 Apache Hadoop 大数据平台生产部署（原汁原味）<a class="headerlink" href="#apache-hadoop" title="Permanent link">&para;</a></h1>
<ul>
<li>相关1：<a href="https://blogs.wl4g.com/archives/3368">基于 CDH 6 大数据平台生产部署</a></li>
</ul>
<p>本文参考了 <a href="https://aws.amazon.com/cn/emr">AWS EMR</a> 和 <a href="https://emr.console.aliyun.com">Aliyun EMR</a> 服务的相关组件部署配置规则，适用于中小规模私有云生产集群手动部署或运维人员技术培训等场景, 另外, a. 如需大规模部署请参考: <a href="https://TODO">基于 absible 的主机 emr 生产集群部署</a>, b. 如需基于 k8s 部署请参考: <a href="https://TODO">基于 kubernetes 的部署 emr 生产集群部署</a></p>
<p>注：以下部署的各组件的版本如有必要请自行修改, 必须确保相关组件版本号的对应关系, 小的版本号改变也可能影响相关组件之间的调用兼容性.</p>
<ul>
<li><a href="https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz">参考 1: Apache Zookeeper 官网下载</a></li>
<li><a href="https://archive.apache.org/dist/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz">参考 2: Apache Hadoop 官网下载</a> =&gt; 注意 x86 32 bits</li>
<li><a href="https://archive.apache.org/dist/hbase/">参考 3: Apache HBase 官网下载</a></li>
<li><a href="http://archive.apache.org/dist/phoenix/apache-phoenix-4.14.1-HBase-1.2/bin/apache-phoenix-4.14.1-HBase-1.2-bin.tar.gz">参考 4: Apache Phoenix 官网下载</a></li>
<li><a href="https://kafka.apache.org/downloads">参考 5: Apache Kafka 官网下载</a></li>
<li><a href="https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz">参考 6: Apache Spark 官网下载</a></li>
<li><a href="https://archive.apache.org/dist/flink/flink-1.14.4/flink-1.14.4-bin-scala_2.11.tgz">参考 7: Apache Flink 官网下载</a></li>
<li><a href="https://github.com/OpenTSDB/opentsdb/releases">参考 8: OpenTSDB 官网下载</a></li>
</ul>
<h2 id="1">1. 部署说明<a class="headerlink" href="#1" title="Permanent link">&para;</a></h2>
<h3 id="11">1.1 资源目录<a class="headerlink" href="#11" title="Permanent link">&para;</a></h3>
<ul>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/emr">/etc/emr/</a></li>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/init.d">/etc/emr/init.d</a></li>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/profile.d">/etc/profile.d</a></li>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/systemd/system">/etc/systemd/system</a></li>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/sysconfig">/etc/sysconfig</a></li>
<li><a href="https://gitee.com/wl4g/blogs/tree/master/docs/hadoop/emr-production-deployment/resources/etc/rc.local">/etc/rc.local</a></li>
<li>...</li>
</ul>
<h3 id="12">1.2 部署拓扑<a class="headerlink" href="#12" title="Permanent link">&para;</a></h3>
<p>为了以最简单最少的服务器完成所有组件的完全分布式部署, 以下 4 台主机(物理机/虚拟机), 在实际机房中部署时请自行按需调整(拆分/隔离), 比如通常 kafka 集群不会部署到 emr 的机器上, 与 emr 的 zookeeper 集群也可以物理隔离(即两个 zookeeper 集群).</p>
<table>
<thead>
<tr>
<th>OS</th>
<th>Hostname</th>
<th>IP</th>
<th>Packages</th>
<th>Processes (jps)</th>
</tr>
</thead>
<tbody>
<tr>
<td>CentOS7.9 8C 32G 500G SSD</td>
<td>emr-header-1</td>
<td>10.0.0.161</td>
<td>open-jdk1.8 / zookeeper-3.4.10 / hadoop-2.7.2 / hbase-1.2.5 / phoenix-4.14.1 / spark-2.3.1-bin-hadoop2.7 / kafka_2.11-1.0.1</td>
<td>QuorumPeerMain / NameNode / JournalNode / DFSZKFailoverController / HistoryServer / WebAppProxyServer / ApplicationHistoryServer / JobHistoryServer / Bootstrap(hdfs httpfs) / ResourceManager / HMaster / ThriftServer / Kafka</td>
</tr>
<tr>
<td>CentOS7.9 8C 32G 500G SSD</td>
<td>emr-header-2</td>
<td>10.0.0.162</td>
<td>open-jdk1.8 / zookeeper-3.4.10 / hadoop-2.7.2 / hbase-1.2.5 / phoenix-4.14.1 / spark-2.3.1-bin-hadoop2.7 / kafka_2.11-1.0.1</td>
<td>QuorumPeerMain / NameNode / JournalNode / DFSZKFailoverController / HistoryServer / WebAppProxyServer / ApplicationHistoryServer / JobHistoryServer / Bootstrap(hdfs httpfs) / ResourceManager / HMaster / ThriftServer / Kafka</td>
</tr>
<tr>
<td>CentOS7.9 4C 32G 4T SSD</td>
<td>emr-worker-1</td>
<td>10.0.0.163</td>
<td>open-jdk1.8 / zookeeper-3.4.10 / hadoop-2.7.2 / hbase-1.2.5 / phoenix-4.14.1 / spark-2.3.1-bin-hadoop2.7 / kafka_2.11-1.0.1</td>
<td>QuorumPeerMain / DataNode / JournalNode / NodeManager / HRegionServer / Kafka / ProdServerStart / TSDMain</td>
</tr>
<tr>
<td>CentOS7.9 4C 32G 4T SSD</td>
<td>emr-worker-2</td>
<td>10.0.0.164</td>
<td>open-jdk1.8 / hadoop-2.7.2 / hbase-1.2.5 / phoenix-4.14.1 / spark-2.3.1-bin-hadoop2.7</td>
<td>DataNode / NodeManager / HRegionServer / TSDMain</td>
</tr>
</tbody>
</table>
<h3 id="12_1">1.2 各组件统一日志目录<a class="headerlink" href="#12_1" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1"></a><a href="#__codelineno-0-1"><span class="linenos" data-linenos="1 "></span></a>/mnt/disk1/log/zookeeper/
<a id="__codelineno-0-2" name="__codelineno-0-2"></a><a href="#__codelineno-0-2"><span class="linenos" data-linenos="2 "></span></a>/mnt/disk1/log/hadoop-hdfs/
<a id="__codelineno-0-3" name="__codelineno-0-3"></a><a href="#__codelineno-0-3"><span class="linenos" data-linenos="3 "></span></a>/mnt/disk1/log/hadoop-yarn/
<a id="__codelineno-0-4" name="__codelineno-0-4"></a><a href="#__codelineno-0-4"><span class="linenos" data-linenos="4 "></span></a>/mnt/disk1/log/hadoop-hbase/
<a id="__codelineno-0-5" name="__codelineno-0-5"></a><a href="#__codelineno-0-5"><span class="linenos" data-linenos="5 "></span></a>/mnt/disk1/log/hadoop-spark/
<a id="__codelineno-0-6" name="__codelineno-0-6"></a><a href="#__codelineno-0-6"><span class="linenos" data-linenos="6 "></span></a>/mnt/disk1/log/spark/
<a id="__codelineno-0-7" name="__codelineno-0-7"></a><a href="#__codelineno-0-7"><span class="linenos" data-linenos="7 "></span></a>/mnt/disk1/log/kafka/
<a id="__codelineno-0-8" name="__codelineno-0-8"></a><a href="#__codelineno-0-8"><span class="linenos" data-linenos="8 "></span></a>/mnt/disk1/log/opentsdb/
</code></pre></div>
<h3 id="13">1.3 各组件统一数据目录<a class="headerlink" href="#13" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1"></a><a href="#__codelineno-1-1"><span class="linenos" data-linenos="1 "></span></a>/mnt/disk1/zookeeper/
<a id="__codelineno-1-2" name="__codelineno-1-2"></a><a href="#__codelineno-1-2"><span class="linenos" data-linenos="2 "></span></a>/mnt/disk1/hadoop/
<a id="__codelineno-1-3" name="__codelineno-1-3"></a><a href="#__codelineno-1-3"><span class="linenos" data-linenos="3 "></span></a>/mnt/disk1/hdfs/
<a id="__codelineno-1-4" name="__codelineno-1-4"></a><a href="#__codelineno-1-4"><span class="linenos" data-linenos="4 "></span></a>/mnt/disk1/yarn/
<a id="__codelineno-1-5" name="__codelineno-1-5"></a><a href="#__codelineno-1-5"><span class="linenos" data-linenos="5 "></span></a>/mnt/disk1/kafka/
<a id="__codelineno-1-6" name="__codelineno-1-6"></a><a href="#__codelineno-1-6"><span class="linenos" data-linenos="6 "></span></a>/mnt/disk1/opentsdb/
</code></pre></div>
<h3 id="14">1.4 其他重要说明<a class="headerlink" href="#14" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p>1.4.1 在部署之前，必须将所有主机按以上 <a href="#1.2_集群服务拓扑">1.2 集群服务拓扑</a> 的规划，提前修改好主机名，且必须做好 <code>emr-header-n</code> 到 <code>emr-worker-n</code> 的 SSH 免密(这里为了简化安全操作，全部使用 root 用户)；</p>
</li>
<li>
<p>1.4.2 以下所有组件手动部署操作过程均在 <code>emr-header-1</code> 节点进行，待所有组件部署完成后(启动之前)，再通过如:
 <code>scp -r /etc/emr/ emr-header-2:/etc/</code> 分发到所有节点，注意 zookeeper 的 <code>/mnt/disk1/zookeeper/myid</code>
 与 kafka 的 <code>/etc/emr/kafka-current/server.properties#broker.id</code> 等状态服务每台机的配置要求不同；</p>
</li>
<li>
<p>1.4.3 有关系统 kernel 优化，请在集群所有组件启动之前配置，<a href="https://blogs.wl4g.com/archives/1294">参考 kernel 调优</a></p>
</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1"></a><a href="#__codelineno-2-1"><span class="linenos" data-linenos="1 "></span></a>sudo cat <span class="s">&lt;&lt;-EOF &gt;&gt;/etc/rc.local</span>
<a id="__codelineno-2-2" name="__codelineno-2-2"></a><a href="#__codelineno-2-2"><span class="linenos" data-linenos="2 "></span></a><span class="s">if test -f /sys/kernel/mm/transparent_hugepage/enabled; then</span>
<a id="__codelineno-2-3" name="__codelineno-2-3"></a><a href="#__codelineno-2-3"><span class="linenos" data-linenos="3 "></span></a><span class="s">   echo never &gt; /sys/kernel/mm/transparent_hugepage/enabled</span>
<a id="__codelineno-2-4" name="__codelineno-2-4"></a><a href="#__codelineno-2-4"><span class="linenos" data-linenos="4 "></span></a><span class="s">fi</span>
<a id="__codelineno-2-5" name="__codelineno-2-5"></a><a href="#__codelineno-2-5"><span class="linenos" data-linenos="5 "></span></a><span class="s">if test -f /sys/kernel/mm/transparent_hugepage/defrag; then</span>
<a id="__codelineno-2-6" name="__codelineno-2-6"></a><a href="#__codelineno-2-6"><span class="linenos" data-linenos="6 "></span></a><span class="s">   echo never &gt; /sys/kernel/mm/transparent_hugepage/defrag</span>
<a id="__codelineno-2-7" name="__codelineno-2-7"></a><a href="#__codelineno-2-7"><span class="linenos" data-linenos="7 "></span></a><span class="s">fi</span>
<a id="__codelineno-2-8" name="__codelineno-2-8"></a><a href="#__codelineno-2-8"><span class="linenos" data-linenos="8 "></span></a><span class="s">EOF</span>
</code></pre></div>
<h2 id="2-zookeeper">2. 部署 Zookeeper 集群<a class="headerlink" href="#2-zookeeper" title="Permanent link">&para;</a></h2>
<blockquote>
<p>注: 每个节点的配置 <code>/mnt/disk1/zookeeper/myid</code> 的值不同, 分别设为: 1,2,3</p>
</blockquote>
<ul>
<li>2.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1"></a><a href="#__codelineno-3-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-3-2" name="__codelineno-3-2"></a><a href="#__codelineno-3-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-3-3" name="__codelineno-3-3"></a><a href="#__codelineno-3-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-3-4" name="__codelineno-3-4"></a><a href="#__codelineno-3-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-zookeeper.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.4.10/zookeeper-3.4.10.tar.gz
<a id="__codelineno-3-5" name="__codelineno-3-5"></a><a href="#__codelineno-3-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-zookeeper.tar.gz
<a id="__codelineno-3-6" name="__codelineno-3-6"></a><a href="#__codelineno-3-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-3-7" name="__codelineno-3-7"></a><a href="#__codelineno-3-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep zookeeper-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/zookeeper-current
</code></pre></div>
<ul>
<li>2.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1"></a><a href="#__codelineno-4-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-zookeeper.sh</span>
<a id="__codelineno-4-2" name="__codelineno-4-2"></a><a href="#__codelineno-4-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-zookeeper.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-zookeeper.sh
<a id="__codelineno-4-3" name="__codelineno-4-3"></a><a href="#__codelineno-4-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-4-4" name="__codelineno-4-4"></a><a href="#__codelineno-4-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-zookeeper.sh
<a id="__codelineno-4-5" name="__codelineno-4-5"></a><a href="#__codelineno-4-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-zookeeper.sh
</code></pre></div>
<ul>
<li>2.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1"></a><a href="#__codelineno-5-1"><span class="linenos" data-linenos="1 "></span></a>mkdir -p /mnt/disk1/zookeeper/data/
<a id="__codelineno-5-2" name="__codelineno-5-2"></a><a href="#__codelineno-5-2"><span class="linenos" data-linenos="2 "></span></a>mkdir -p /etc/emr/zookeeper-conf
<a id="__codelineno-5-3" name="__codelineno-5-3"></a><a href="#__codelineno-5-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-5-4" name="__codelineno-5-4"></a><a href="#__codelineno-5-4"><span class="linenos" data-linenos="4 "></span></a><span class="c1"># Generating zoo.cfg</span>
<a id="__codelineno-5-5" name="__codelineno-5-5"></a><a href="#__codelineno-5-5"><span class="linenos" data-linenos="5 "></span></a>curl -Lk -o <span class="nv">$ZOOCFGDIR</span>/zoo.cfg https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/zookeeper-conf/zoo.cfg
</code></pre></div>
<ul>
<li>2.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1"></a><a href="#__codelineno-6-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating zookeeper.service</span>
<a id="__codelineno-6-2" name="__codelineno-6-2"></a><a href="#__codelineno-6-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o <span class="nv">$ZOOCFGDIR</span>/zoo.cfg https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/zookeeper.service
</code></pre></div>
<h2 id="3-hadoop">3. 部署 Hadoop 集群<a class="headerlink" href="#3-hadoop" title="Permanent link">&para;</a></h2>
<ul>
<li>3.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1"></a><a href="#__codelineno-7-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># TODO</span>
</code></pre></div>
<ul>
<li>3.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1"></a><a href="#__codelineno-8-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-hdfs.sh</span>
<a id="__codelineno-8-2" name="__codelineno-8-2"></a><a href="#__codelineno-8-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-hdfs.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-hdfs.sh
<a id="__codelineno-8-3" name="__codelineno-8-3"></a><a href="#__codelineno-8-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-8-4" name="__codelineno-8-4"></a><a href="#__codelineno-8-4"><span class="linenos" data-linenos="4 "></span></a><span class="c1"># Generating profile-yarn.sh</span>
<a id="__codelineno-8-5" name="__codelineno-8-5"></a><a href="#__codelineno-8-5"><span class="linenos" data-linenos="5 "></span></a>curl -Lk -o /etc/profile.d/profile-yarn.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-yarn.sh
</code></pre></div>
<ul>
<li>3.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1"></a><a href="#__codelineno-9-1"><span class="linenos" data-linenos=" 1 "></span></a>mkdir -p <span class="nv">$HADOOP_LOG_DIR</span>
<a id="__codelineno-9-2" name="__codelineno-9-2"></a><a href="#__codelineno-9-2"><span class="linenos" data-linenos=" 2 "></span></a>mkdir -p <span class="nv">$YARN_LOG_DIR</span>
<a id="__codelineno-9-3" name="__codelineno-9-3"></a><a href="#__codelineno-9-3"><span class="linenos" data-linenos=" 3 "></span></a>mkdir -p <span class="nv">$HADOOP_CONF_DIR</span>
<a id="__codelineno-9-4" name="__codelineno-9-4"></a><a href="#__codelineno-9-4"><span class="linenos" data-linenos=" 4 "></span></a>cp -r <span class="nv">$HADOOP_HOME</span>/etc/hadoop/ <span class="nv">$HADOOP_CONF_DIR</span>
<a id="__codelineno-9-5" name="__codelineno-9-5"></a><a href="#__codelineno-9-5"><span class="linenos" data-linenos=" 5 "></span></a>
<a id="__codelineno-9-6" name="__codelineno-9-6"></a><a href="#__codelineno-9-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="c1"># Generating hadoop-env.sh</span>
<a id="__codelineno-9-7" name="__codelineno-9-7"></a><a href="#__codelineno-9-7"><span class="linenos" data-linenos=" 7 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/hadoop-env.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/hadoop-env.sh
<a id="__codelineno-9-8" name="__codelineno-9-8"></a><a href="#__codelineno-9-8"><span class="linenos" data-linenos=" 8 "></span></a>
<a id="__codelineno-9-9" name="__codelineno-9-9"></a><a href="#__codelineno-9-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="c1"># Generating core-site.xml</span>
<a id="__codelineno-9-10" name="__codelineno-9-10"></a><a href="#__codelineno-9-10"><span class="linenos" data-linenos="10 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/core-site.xml https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/core-site.xml
<a id="__codelineno-9-11" name="__codelineno-9-11"></a><a href="#__codelineno-9-11"><span class="linenos" data-linenos="11 "></span></a>
<a id="__codelineno-9-12" name="__codelineno-9-12"></a><a href="#__codelineno-9-12"><span class="linenos" data-linenos="12 "></span></a><span class="c1"># Generating hdfs-site.xml</span>
<a id="__codelineno-9-13" name="__codelineno-9-13"></a><a href="#__codelineno-9-13"><span class="linenos" data-linenos="13 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/hdfs-site.xml https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/hdfs-site.xml
<a id="__codelineno-9-14" name="__codelineno-9-14"></a><a href="#__codelineno-9-14"><span class="linenos" data-linenos="14 "></span></a>
<a id="__codelineno-9-15" name="__codelineno-9-15"></a><a href="#__codelineno-9-15"><span class="linenos" data-linenos="15 "></span></a><span class="c1"># Generating mapred-env.sh</span>
<a id="__codelineno-9-16" name="__codelineno-9-16"></a><a href="#__codelineno-9-16"><span class="linenos" data-linenos="16 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/mapred-env.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/mapred-env.sh
<a id="__codelineno-9-17" name="__codelineno-9-17"></a><a href="#__codelineno-9-17"><span class="linenos" data-linenos="17 "></span></a>
<a id="__codelineno-9-18" name="__codelineno-9-18"></a><a href="#__codelineno-9-18"><span class="linenos" data-linenos="18 "></span></a><span class="c1"># Generating mapred-site.xml</span>
<a id="__codelineno-9-19" name="__codelineno-9-19"></a><a href="#__codelineno-9-19"><span class="linenos" data-linenos="19 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/mapred-site.xml https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/mapred-site.xml
<a id="__codelineno-9-20" name="__codelineno-9-20"></a><a href="#__codelineno-9-20"><span class="linenos" data-linenos="20 "></span></a>
<a id="__codelineno-9-21" name="__codelineno-9-21"></a><a href="#__codelineno-9-21"><span class="linenos" data-linenos="21 "></span></a><span class="c1"># Generating yarn-env.sh</span>
<a id="__codelineno-9-22" name="__codelineno-9-22"></a><a href="#__codelineno-9-22"><span class="linenos" data-linenos="22 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/yarn-env.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/yarn-env.sh
<a id="__codelineno-9-23" name="__codelineno-9-23"></a><a href="#__codelineno-9-23"><span class="linenos" data-linenos="23 "></span></a>
<a id="__codelineno-9-24" name="__codelineno-9-24"></a><a href="#__codelineno-9-24"><span class="linenos" data-linenos="24 "></span></a><span class="c1"># Generating yarn-site.xml</span>
<a id="__codelineno-9-25" name="__codelineno-9-25"></a><a href="#__codelineno-9-25"><span class="linenos" data-linenos="25 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/yarn-site.xml https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hadoop-conf/yarn-site.xml
<a id="__codelineno-9-26" name="__codelineno-9-26"></a><a href="#__codelineno-9-26"><span class="linenos" data-linenos="26 "></span></a>
<a id="__codelineno-9-27" name="__codelineno-9-27"></a><a href="#__codelineno-9-27"><span class="linenos" data-linenos="27 "></span></a><span class="c1"># Generating slaves</span>
<a id="__codelineno-9-28" name="__codelineno-9-28"></a><a href="#__codelineno-9-28"><span class="linenos" data-linenos="28 "></span></a>cat <span class="s">&lt;&lt;-&#39;EOF&#39;&gt; /etc/emr/hadoop-conf/slaves</span>
<a id="__codelineno-9-29" name="__codelineno-9-29"></a><a href="#__codelineno-9-29"><span class="linenos" data-linenos="29 "></span></a><span class="s">emr-worker-1</span>
<a id="__codelineno-9-30" name="__codelineno-9-30"></a><a href="#__codelineno-9-30"><span class="linenos" data-linenos="30 "></span></a><span class="s">emr-worker-2</span>
<a id="__codelineno-9-31" name="__codelineno-9-31"></a><a href="#__codelineno-9-31"><span class="linenos" data-linenos="31 "></span></a><span class="s">EOF</span>
</code></pre></div>
<ul>
<li>3.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-10-1" name="__codelineno-10-1"></a><a href="#__codelineno-10-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="nb">cd</span> /etc/systemd/system
<a id="__codelineno-10-2" name="__codelineno-10-2"></a><a href="#__codelineno-10-2"><span class="linenos" data-linenos=" 2 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-namenode.service
<a id="__codelineno-10-3" name="__codelineno-10-3"></a><a href="#__codelineno-10-3"><span class="linenos" data-linenos=" 3 "></span></a>
<a id="__codelineno-10-4" name="__codelineno-10-4"></a><a href="#__codelineno-10-4"><span class="linenos" data-linenos=" 4 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-journalnode.service
<a id="__codelineno-10-5" name="__codelineno-10-5"></a><a href="#__codelineno-10-5"><span class="linenos" data-linenos=" 5 "></span></a>
<a id="__codelineno-10-6" name="__codelineno-10-6"></a><a href="#__codelineno-10-6"><span class="linenos" data-linenos=" 6 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-zkfc.service
<a id="__codelineno-10-7" name="__codelineno-10-7"></a><a href="#__codelineno-10-7"><span class="linenos" data-linenos=" 7 "></span></a>
<a id="__codelineno-10-8" name="__codelineno-10-8"></a><a href="#__codelineno-10-8"><span class="linenos" data-linenos=" 8 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-resourcemanager.service
<a id="__codelineno-10-9" name="__codelineno-10-9"></a><a href="#__codelineno-10-9"><span class="linenos" data-linenos=" 9 "></span></a>
<a id="__codelineno-10-10" name="__codelineno-10-10"></a><a href="#__codelineno-10-10"><span class="linenos" data-linenos="10 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-httpfs.service
<a id="__codelineno-10-11" name="__codelineno-10-11"></a><a href="#__codelineno-10-11"><span class="linenos" data-linenos="11 "></span></a>
<a id="__codelineno-10-12" name="__codelineno-10-12"></a><a href="#__codelineno-10-12"><span class="linenos" data-linenos="12 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-historyserver.service
<a id="__codelineno-10-13" name="__codelineno-10-13"></a><a href="#__codelineno-10-13"><span class="linenos" data-linenos="13 "></span></a>
<a id="__codelineno-10-14" name="__codelineno-10-14"></a><a href="#__codelineno-10-14"><span class="linenos" data-linenos="14 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-nodemanager.service
<a id="__codelineno-10-15" name="__codelineno-10-15"></a><a href="#__codelineno-10-15"><span class="linenos" data-linenos="15 "></span></a>
<a id="__codelineno-10-16" name="__codelineno-10-16"></a><a href="#__codelineno-10-16"><span class="linenos" data-linenos="16 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hadoop-datanode.service
</code></pre></div>
<h2 id="4-hbase">4. 部署 HBase 集群<a class="headerlink" href="#4-hbase" title="Permanent link">&para;</a></h2>
<ul>
<li>4.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-11-1" name="__codelineno-11-1"></a><a href="#__codelineno-11-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-11-2" name="__codelineno-11-2"></a><a href="#__codelineno-11-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-11-3" name="__codelineno-11-3"></a><a href="#__codelineno-11-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-11-4" name="__codelineno-11-4"></a><a href="#__codelineno-11-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-hbase.tgz https://archive.apache.org/dist/hbase/1.2.5/hbase-1.2.5-bin.tar.gz
<a id="__codelineno-11-5" name="__codelineno-11-5"></a><a href="#__codelineno-11-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-hbase.tgz
<a id="__codelineno-11-6" name="__codelineno-11-6"></a><a href="#__codelineno-11-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-11-7" name="__codelineno-11-7"></a><a href="#__codelineno-11-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep hbase-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/hbase-current
</code></pre></div>
<ul>
<li>4.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-12-1" name="__codelineno-12-1"></a><a href="#__codelineno-12-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-hbase.sh</span>
<a id="__codelineno-12-2" name="__codelineno-12-2"></a><a href="#__codelineno-12-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-hbase.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-hbase.sh
<a id="__codelineno-12-3" name="__codelineno-12-3"></a><a href="#__codelineno-12-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-12-4" name="__codelineno-12-4"></a><a href="#__codelineno-12-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-hbase.sh
<a id="__codelineno-12-5" name="__codelineno-12-5"></a><a href="#__codelineno-12-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-hbase.sh
</code></pre></div>
<ul>
<li>4.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-13-1" name="__codelineno-13-1"></a><a href="#__codelineno-13-1"><span class="linenos" data-linenos=" 1 "></span></a>mkdir -p <span class="nv">$HBASE_LOG_DIR</span>
<a id="__codelineno-13-2" name="__codelineno-13-2"></a><a href="#__codelineno-13-2"><span class="linenos" data-linenos=" 2 "></span></a>mkdir -p <span class="nv">$HBASE_CONF_DIR</span>
<a id="__codelineno-13-3" name="__codelineno-13-3"></a><a href="#__codelineno-13-3"><span class="linenos" data-linenos=" 3 "></span></a>cp -r <span class="nv">$HBASE_HOME</span>/conf/ <span class="nv">$HBASE_CONF_DIR</span>
<a id="__codelineno-13-4" name="__codelineno-13-4"></a><a href="#__codelineno-13-4"><span class="linenos" data-linenos=" 4 "></span></a>
<a id="__codelineno-13-5" name="__codelineno-13-5"></a><a href="#__codelineno-13-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="c1"># Generating hbase-env.sh</span>
<a id="__codelineno-13-6" name="__codelineno-13-6"></a><a href="#__codelineno-13-6"><span class="linenos" data-linenos=" 6 "></span></a>curl -Lk -o <span class="nv">$HBASE_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hbase-conf/hbase-env.sh
<a id="__codelineno-13-7" name="__codelineno-13-7"></a><a href="#__codelineno-13-7"><span class="linenos" data-linenos=" 7 "></span></a>
<a id="__codelineno-13-8" name="__codelineno-13-8"></a><a href="#__codelineno-13-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="c1"># Generating hbase-site.xml</span>
<a id="__codelineno-13-9" name="__codelineno-13-9"></a><a href="#__codelineno-13-9"><span class="linenos" data-linenos=" 9 "></span></a>curl -Lk -o <span class="nv">$HBASE_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/hbase-conf/hbase-site.xml
<a id="__codelineno-13-10" name="__codelineno-13-10"></a><a href="#__codelineno-13-10"><span class="linenos" data-linenos="10 "></span></a>
<a id="__codelineno-13-11" name="__codelineno-13-11"></a><a href="#__codelineno-13-11"><span class="linenos" data-linenos="11 "></span></a><span class="c1"># Generating regionservers</span>
<a id="__codelineno-13-12" name="__codelineno-13-12"></a><a href="#__codelineno-13-12"><span class="linenos" data-linenos="12 "></span></a>cat <span class="s">&lt;&lt;-&#39;EOF&#39;&gt; /etc/emr/hbase-conf/regionservers</span>
<a id="__codelineno-13-13" name="__codelineno-13-13"></a><a href="#__codelineno-13-13"><span class="linenos" data-linenos="13 "></span></a><span class="s">emr-worker-1</span>
<a id="__codelineno-13-14" name="__codelineno-13-14"></a><a href="#__codelineno-13-14"><span class="linenos" data-linenos="14 "></span></a><span class="s">emr-worker-2</span>
<a id="__codelineno-13-15" name="__codelineno-13-15"></a><a href="#__codelineno-13-15"><span class="linenos" data-linenos="15 "></span></a><span class="s">EOF</span>
</code></pre></div>
<ul>
<li>4.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-14-1" name="__codelineno-14-1"></a><a href="#__codelineno-14-1"><span class="linenos" data-linenos="1 "></span></a><span class="nb">cd</span> /etc/systemd/system
<a id="__codelineno-14-2" name="__codelineno-14-2"></a><a href="#__codelineno-14-2"><span class="linenos" data-linenos="2 "></span></a>
<a id="__codelineno-14-3" name="__codelineno-14-3"></a><a href="#__codelineno-14-3"><span class="linenos" data-linenos="3 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hbase-hmaster.service
<a id="__codelineno-14-4" name="__codelineno-14-4"></a><a href="#__codelineno-14-4"><span class="linenos" data-linenos="4 "></span></a>
<a id="__codelineno-14-5" name="__codelineno-14-5"></a><a href="#__codelineno-14-5"><span class="linenos" data-linenos="5 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hbase-regionserver.service
<a id="__codelineno-14-6" name="__codelineno-14-6"></a><a href="#__codelineno-14-6"><span class="linenos" data-linenos="6 "></span></a>
<a id="__codelineno-14-7" name="__codelineno-14-7"></a><a href="#__codelineno-14-7"><span class="linenos" data-linenos="7 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/hbase-thrift.service
</code></pre></div>
<h2 id="5-phoenix">5. 集成 Phoenix 到集群<a class="headerlink" href="#5-phoenix" title="Permanent link">&para;</a></h2>
<ul>
<li>5.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-15-1" name="__codelineno-15-1"></a><a href="#__codelineno-15-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-15-2" name="__codelineno-15-2"></a><a href="#__codelineno-15-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-15-3" name="__codelineno-15-3"></a><a href="#__codelineno-15-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-15-4" name="__codelineno-15-4"></a><a href="#__codelineno-15-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-phoenix.tgz http://archive.apache.org/dist/phoenix/apache-phoenix-4.14.1-HBase-1.2/bin/apache-phoenix-4.14.1-HBase-1.2-bin.tar.gz
<a id="__codelineno-15-5" name="__codelineno-15-5"></a><a href="#__codelineno-15-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-phoenix.tgz
<a id="__codelineno-15-6" name="__codelineno-15-6"></a><a href="#__codelineno-15-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-15-7" name="__codelineno-15-7"></a><a href="#__codelineno-15-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep phoenix-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/phoenix-current
</code></pre></div>
<ul>
<li>5.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-16-1" name="__codelineno-16-1"></a><a href="#__codelineno-16-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-phoenix.sh</span>
<a id="__codelineno-16-2" name="__codelineno-16-2"></a><a href="#__codelineno-16-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-phoenix.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-phoenix.sh
<a id="__codelineno-16-3" name="__codelineno-16-3"></a><a href="#__codelineno-16-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-16-4" name="__codelineno-16-4"></a><a href="#__codelineno-16-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-phoenix.sh
<a id="__codelineno-16-5" name="__codelineno-16-5"></a><a href="#__codelineno-16-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-phoenix.sh
</code></pre></div>
<ul>
<li>5.3 关联 HBase</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-17-1" name="__codelineno-17-1"></a><a href="#__codelineno-17-1"><span class="linenos" data-linenos="1 "></span></a>cp -r <span class="nv">$PHOENIX_HOME</span>/phoenix-*-*-*-server.jar <span class="nv">$HBASE_HOME</span>/lib/
</code></pre></div>
<h2 id="5-spark">5. 部署 Spark 集群<a class="headerlink" href="#5-spark" title="Permanent link">&para;</a></h2>
<ul>
<li>5.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-18-1" name="__codelineno-18-1"></a><a href="#__codelineno-18-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-18-2" name="__codelineno-18-2"></a><a href="#__codelineno-18-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-18-3" name="__codelineno-18-3"></a><a href="#__codelineno-18-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-18-4" name="__codelineno-18-4"></a><a href="#__codelineno-18-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-spark.tgz https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz
<a id="__codelineno-18-5" name="__codelineno-18-5"></a><a href="#__codelineno-18-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-spark.tgz
<a id="__codelineno-18-6" name="__codelineno-18-6"></a><a href="#__codelineno-18-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-18-7" name="__codelineno-18-7"></a><a href="#__codelineno-18-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep spark-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/spark-current
</code></pre></div>
<ul>
<li>5.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-19-1" name="__codelineno-19-1"></a><a href="#__codelineno-19-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-spark.sh</span>
<a id="__codelineno-19-2" name="__codelineno-19-2"></a><a href="#__codelineno-19-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-spark.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/spark-conf/profile-spark.sh
<a id="__codelineno-19-3" name="__codelineno-19-3"></a><a href="#__codelineno-19-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-19-4" name="__codelineno-19-4"></a><a href="#__codelineno-19-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-spark.sh
<a id="__codelineno-19-5" name="__codelineno-19-5"></a><a href="#__codelineno-19-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-spark.sh
</code></pre></div>
<ul>
<li>5.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-20-1" name="__codelineno-20-1"></a><a href="#__codelineno-20-1"><span class="linenos" data-linenos=" 1 "></span></a>mkdir -p <span class="nv">$SPARK_LOG_DIR</span>
<a id="__codelineno-20-2" name="__codelineno-20-2"></a><a href="#__codelineno-20-2"><span class="linenos" data-linenos=" 2 "></span></a>mkdir -p <span class="nv">$SPARK_CONF_DIR</span>
<a id="__codelineno-20-3" name="__codelineno-20-3"></a><a href="#__codelineno-20-3"><span class="linenos" data-linenos=" 3 "></span></a>cp -r <span class="nv">$SPARK_HOME</span>/conf/ <span class="nv">$SPARK_CONF_DIR</span>
<a id="__codelineno-20-4" name="__codelineno-20-4"></a><a href="#__codelineno-20-4"><span class="linenos" data-linenos=" 4 "></span></a>
<a id="__codelineno-20-5" name="__codelineno-20-5"></a><a href="#__codelineno-20-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="c1"># Generating spark-defaults.conf</span>
<a id="__codelineno-20-6" name="__codelineno-20-6"></a><a href="#__codelineno-20-6"><span class="linenos" data-linenos=" 6 "></span></a>curl -Lk -o <span class="nv">$SPARK_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/spark-conf/spark-defaults.conf
<a id="__codelineno-20-7" name="__codelineno-20-7"></a><a href="#__codelineno-20-7"><span class="linenos" data-linenos=" 7 "></span></a>
<a id="__codelineno-20-8" name="__codelineno-20-8"></a><a href="#__codelineno-20-8"><span class="linenos" data-linenos=" 8 "></span></a><span class="c1"># Generating spark-env.sh</span>
<a id="__codelineno-20-9" name="__codelineno-20-9"></a><a href="#__codelineno-20-9"><span class="linenos" data-linenos=" 9 "></span></a>curl -Lk -o <span class="nv">$SPARK_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/spark-conf/spark-env.sh
<a id="__codelineno-20-10" name="__codelineno-20-10"></a><a href="#__codelineno-20-10"><span class="linenos" data-linenos="10 "></span></a>
<a id="__codelineno-20-11" name="__codelineno-20-11"></a><a href="#__codelineno-20-11"><span class="linenos" data-linenos="11 "></span></a><span class="c1"># Generating hive-site.xml</span>
<a id="__codelineno-20-12" name="__codelineno-20-12"></a><a href="#__codelineno-20-12"><span class="linenos" data-linenos="12 "></span></a>curl -Lk -o <span class="nv">$SPARK_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/spark-conf/hive-site.xml
</code></pre></div>
<ul>
<li>5.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-21-1" name="__codelineno-21-1"></a><a href="#__codelineno-21-1"><span class="linenos" data-linenos="1 "></span></a><span class="nb">cd</span> /etc/systemd/system
<a id="__codelineno-21-2" name="__codelineno-21-2"></a><a href="#__codelineno-21-2"><span class="linenos" data-linenos="2 "></span></a>
<a id="__codelineno-21-3" name="__codelineno-21-3"></a><a href="#__codelineno-21-3"><span class="linenos" data-linenos="3 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/spark-historyserver.service
</code></pre></div>
<h2 id="6-flink">6. 部署 Flink 集群<a class="headerlink" href="#6-flink" title="Permanent link">&para;</a></h2>
<ul>
<li>6.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-22-1" name="__codelineno-22-1"></a><a href="#__codelineno-22-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-22-2" name="__codelineno-22-2"></a><a href="#__codelineno-22-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-22-3" name="__codelineno-22-3"></a><a href="#__codelineno-22-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-22-4" name="__codelineno-22-4"></a><a href="#__codelineno-22-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-flink.tgz https://archive.apache.org/dist/flink/flink-1.14.4/flink-1.14.4-bin-scala_2.11.tgz
<a id="__codelineno-22-5" name="__codelineno-22-5"></a><a href="#__codelineno-22-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-flink.tgz
<a id="__codelineno-22-6" name="__codelineno-22-6"></a><a href="#__codelineno-22-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-22-7" name="__codelineno-22-7"></a><a href="#__codelineno-22-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep flink-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/flink-current
</code></pre></div>
<ul>
<li>6.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-23-1" name="__codelineno-23-1"></a><a href="#__codelineno-23-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-flink.sh</span>
<a id="__codelineno-23-2" name="__codelineno-23-2"></a><a href="#__codelineno-23-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-flink.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-flink.sh
<a id="__codelineno-23-3" name="__codelineno-23-3"></a><a href="#__codelineno-23-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-23-4" name="__codelineno-23-4"></a><a href="#__codelineno-23-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-flink.sh
<a id="__codelineno-23-5" name="__codelineno-23-5"></a><a href="#__codelineno-23-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-flink.sh
</code></pre></div>
<ul>
<li>6.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-24-1" name="__codelineno-24-1"></a><a href="#__codelineno-24-1"><span class="linenos" data-linenos="1 "></span></a>mkdir -p <span class="nv">$FLINK_LOG_DIR</span>
<a id="__codelineno-24-2" name="__codelineno-24-2"></a><a href="#__codelineno-24-2"><span class="linenos" data-linenos="2 "></span></a>mkdir -p <span class="nv">$FLINK_CONF_DIR</span>
<a id="__codelineno-24-3" name="__codelineno-24-3"></a><a href="#__codelineno-24-3"><span class="linenos" data-linenos="3 "></span></a>cp -r <span class="nv">$FLINK_HOME</span>/conf/ <span class="nv">$FLINK_CONF_DIR</span>
<a id="__codelineno-24-4" name="__codelineno-24-4"></a><a href="#__codelineno-24-4"><span class="linenos" data-linenos="4 "></span></a>
<a id="__codelineno-24-5" name="__codelineno-24-5"></a><a href="#__codelineno-24-5"><span class="linenos" data-linenos="5 "></span></a><span class="c1"># Generating flink-conf.yaml</span>
<a id="__codelineno-24-6" name="__codelineno-24-6"></a><a href="#__codelineno-24-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># TODO</span>
</code></pre></div>
<ul>
<li>6.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-25-1" name="__codelineno-25-1"></a><a href="#__codelineno-25-1"><span class="linenos" data-linenos="1 "></span></a><span class="nb">cd</span> /etc/systemd/system
<a id="__codelineno-25-2" name="__codelineno-25-2"></a><a href="#__codelineno-25-2"><span class="linenos" data-linenos="2 "></span></a>
<a id="__codelineno-25-3" name="__codelineno-25-3"></a><a href="#__codelineno-25-3"><span class="linenos" data-linenos="3 "></span></a>curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/flink-standalone.service
</code></pre></div>
<h2 id="7-kafka">7. 部署 Kafka 集群<a class="headerlink" href="#7-kafka" title="Permanent link">&para;</a></h2>
<blockquote>
<p>注: 每个节点的配置 <code>/etc/emr/kafka-conf/server.properties#broker.id</code> 的值不同, 分别设为: 1,2,3</p>
</blockquote>
<ul>
<li>7.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-26-1" name="__codelineno-26-1"></a><a href="#__codelineno-26-1"><span class="linenos" data-linenos="1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-26-2" name="__codelineno-26-2"></a><a href="#__codelineno-26-2"><span class="linenos" data-linenos="2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-26-3" name="__codelineno-26-3"></a><a href="#__codelineno-26-3"><span class="linenos" data-linenos="3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-26-4" name="__codelineno-26-4"></a><a href="#__codelineno-26-4"><span class="linenos" data-linenos="4 "></span></a>wget -O apache-kafka.tgz https://archive.apache.org/dist/kafka/1.0.1/kafka_2.11-1.0.1.tgz
<a id="__codelineno-26-5" name="__codelineno-26-5"></a><a href="#__codelineno-26-5"><span class="linenos" data-linenos="5 "></span></a>tar -xf apache-kafka.tgz
<a id="__codelineno-26-6" name="__codelineno-26-6"></a><a href="#__codelineno-26-6"><span class="linenos" data-linenos="6 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-26-7" name="__codelineno-26-7"></a><a href="#__codelineno-26-7"><span class="linenos" data-linenos="7 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep kafka_<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/kafka-current
</code></pre></div>
<ul>
<li>7.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-27-1" name="__codelineno-27-1"></a><a href="#__codelineno-27-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-kafka.sh</span>
<a id="__codelineno-27-2" name="__codelineno-27-2"></a><a href="#__codelineno-27-2"><span class="linenos" data-linenos="2 "></span></a>sudo curl -Lk -o /etc/profile.d/profile-kafka.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-kafka.sh
<a id="__codelineno-27-3" name="__codelineno-27-3"></a><a href="#__codelineno-27-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-27-4" name="__codelineno-27-4"></a><a href="#__codelineno-27-4"><span class="linenos" data-linenos="4 "></span></a>sudo chmod +x /etc/profile.d/profile-kafka.sh
<a id="__codelineno-27-5" name="__codelineno-27-5"></a><a href="#__codelineno-27-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-kafka.sh
</code></pre></div>
<ul>
<li>7.3 运行配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-28-1" name="__codelineno-28-1"></a><a href="#__codelineno-28-1"><span class="linenos" data-linenos="1 "></span></a>mkdir -p <span class="nv">$KAFKA_CONF_DIR</span>
<a id="__codelineno-28-2" name="__codelineno-28-2"></a><a href="#__codelineno-28-2"><span class="linenos" data-linenos="2 "></span></a>mkdir -p <span class="nv">$KAFKA_DATA_DIR</span>
<a id="__codelineno-28-3" name="__codelineno-28-3"></a><a href="#__codelineno-28-3"><span class="linenos" data-linenos="3 "></span></a>cp -r /usr/lib/kafka-current/config/* <span class="nv">$KAFKA_CONF_DIR</span>
<a id="__codelineno-28-4" name="__codelineno-28-4"></a><a href="#__codelineno-28-4"><span class="linenos" data-linenos="4 "></span></a>
<a id="__codelineno-28-5" name="__codelineno-28-5"></a><a href="#__codelineno-28-5"><span class="linenos" data-linenos="5 "></span></a><span class="c1"># Generating server.properties</span>
<a id="__codelineno-28-6" name="__codelineno-28-6"></a><a href="#__codelineno-28-6"><span class="linenos" data-linenos="6 "></span></a>sudo curl -Lk -o <span class="nv">$KAFKA_CONF_DIR</span> https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr/kafka-conf/server.properties
</code></pre></div>
<ul>
<li>7.4 服务配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-29-1" name="__codelineno-29-1"></a><a href="#__codelineno-29-1"><span class="linenos" data-linenos="1 "></span></a><span class="nb">cd</span> /etc/systemd/system
<a id="__codelineno-29-2" name="__codelineno-29-2"></a><a href="#__codelineno-29-2"><span class="linenos" data-linenos="2 "></span></a>
<a id="__codelineno-29-3" name="__codelineno-29-3"></a><a href="#__codelineno-29-3"><span class="linenos" data-linenos="3 "></span></a>sudo curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/kafka.service
</code></pre></div>
<ul>
<li>7.5 Kafka Manager 裸机部署</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-30-1" name="__codelineno-30-1"></a><a href="#__codelineno-30-1"><span class="linenos" data-linenos="1 "></span></a>sudo curl -Lk -O https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/systemd/system/kafka-manager.service
<a id="__codelineno-30-2" name="__codelineno-30-2"></a><a href="#__codelineno-30-2"><span class="linenos" data-linenos="2 "></span></a>
<a id="__codelineno-30-3" name="__codelineno-30-3"></a><a href="#__codelineno-30-3"><span class="linenos" data-linenos="3 "></span></a>sudo curl -Lk -o /etc/init.d/kafka-manager.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/init.d/kafka-manager.sh
<a id="__codelineno-30-4" name="__codelineno-30-4"></a><a href="#__codelineno-30-4"><span class="linenos" data-linenos="4 "></span></a>sudo chmod +x /etc/init.d/kafka-manager.sh
<a id="__codelineno-30-5" name="__codelineno-30-5"></a><a href="#__codelineno-30-5"><span class="linenos" data-linenos="5 "></span></a>
<a id="__codelineno-30-6" name="__codelineno-30-6"></a><a href="#__codelineno-30-6"><span class="linenos" data-linenos="6 "></span></a>sudo mkdir -p <span class="nv">$KAFKA_MANAGER_HOME</span>
<a id="__codelineno-30-7" name="__codelineno-30-7"></a><a href="#__codelineno-30-7"><span class="linenos" data-linenos="7 "></span></a>sudo ln -snf /opt/apps/kafka-manager-1.3.3.16 <span class="nv">$KAFKA_MANAGER_HOME</span>
</code></pre></div>
<ul>
<li>7.6 (推荐) Kafka Manager docker 部署</li>
</ul>
<blockquote>
<p>注：kafka-manager-2.x 版连接 kafka-2.x，kafka-1.x 用不了</p>
</blockquote>
<div class="highlight"><pre><span></span><code><a id="__codelineno-31-1" name="__codelineno-31-1"></a><a href="#__codelineno-31-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># 连接本地 zookeeper</span>
<a id="__codelineno-31-2" name="__codelineno-31-2"></a><a href="#__codelineno-31-2"><span class="linenos" data-linenos="2 "></span></a>docker run --rm --name<span class="o">=</span>kafka-manager1 --network<span class="o">=</span>host -e <span class="nv">ZK_HOSTS</span><span class="o">=</span><span class="m">127</span>.0.0.1:2181 registry.cn-shenzhen.aliyuncs.com/wl4g/kafka-manager:2.0.0.2
</code></pre></div>
<h2 id="8-opentsdb">8. 部署 OpenTSDB 集群<a class="headerlink" href="#8-opentsdb" title="Permanent link">&para;</a></h2>
<ul>
<li>8.1 安装准备</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-32-1" name="__codelineno-32-1"></a><a href="#__codelineno-32-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="nv">BASE_DIR</span><span class="o">=</span>/opt/apps/emr
<a id="__codelineno-32-2" name="__codelineno-32-2"></a><a href="#__codelineno-32-2"><span class="linenos" data-linenos=" 2 "></span></a><span class="c1"># Download</span>
<a id="__codelineno-32-3" name="__codelineno-32-3"></a><a href="#__codelineno-32-3"><span class="linenos" data-linenos=" 3 "></span></a>mkdir -p <span class="nv">$BASE_DIR</span><span class="p">;</span> <span class="nb">cd</span> <span class="nv">$BASE_DIR</span>
<a id="__codelineno-32-4" name="__codelineno-32-4"></a><a href="#__codelineno-32-4"><span class="linenos" data-linenos=" 4 "></span></a>wget -O apache-opentsdb.tgz https://github.com/OpenTSDB/opentsdb/releases/download/v2.4.1/opentsdb-2.4.1-1-20210902183110-root.noarch.rpm
<a id="__codelineno-32-5" name="__codelineno-32-5"></a><a href="#__codelineno-32-5"><span class="linenos" data-linenos=" 5 "></span></a><span class="c1"># or</span>
<a id="__codelineno-32-6" name="__codelineno-32-6"></a><a href="#__codelineno-32-6"><span class="linenos" data-linenos=" 6 "></span></a><span class="c1">#wget -O apache-opentsdb.tgz https://github.com/OpenTSDB/opentsdb/releases/download/v2.4.1/opentsdb-2.4.1_all.deb</span>
<a id="__codelineno-32-7" name="__codelineno-32-7"></a><a href="#__codelineno-32-7"><span class="linenos" data-linenos=" 7 "></span></a>
<a id="__codelineno-32-8" name="__codelineno-32-8"></a><a href="#__codelineno-32-8"><span class="linenos" data-linenos=" 8 "></span></a>tar -xf apache-opentsdb.tgz
<a id="__codelineno-32-9" name="__codelineno-32-9"></a><a href="#__codelineno-32-9"><span class="linenos" data-linenos=" 9 "></span></a><span class="c1"># Add soft link</span>
<a id="__codelineno-32-10" name="__codelineno-32-10"></a><a href="#__codelineno-32-10"><span class="linenos" data-linenos="10 "></span></a>ln -snf <span class="k">$(</span><span class="nb">cd</span> <span class="sb">`</span>ls <span class="p">|</span> grep opentsdb-<span class="sb">`</span><span class="p">;</span><span class="nb">pwd</span><span class="k">)</span> /usr/lib/opentsdb-current
</code></pre></div>
<ul>
<li>8.2 环境配置</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-33-1" name="__codelineno-33-1"></a><a href="#__codelineno-33-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># Generating profile-opentsdb.sh</span>
<a id="__codelineno-33-2" name="__codelineno-33-2"></a><a href="#__codelineno-33-2"><span class="linenos" data-linenos="2 "></span></a>curl -Lk -o /etc/profile.d/profile-opentsdb.sh https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/profile.d/profile-opentsdb.sh
<a id="__codelineno-33-3" name="__codelineno-33-3"></a><a href="#__codelineno-33-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-33-4" name="__codelineno-33-4"></a><a href="#__codelineno-33-4"><span class="linenos" data-linenos="4 "></span></a>chmod +x /etc/profile.d/profile-opentsdb.sh
<a id="__codelineno-33-5" name="__codelineno-33-5"></a><a href="#__codelineno-33-5"><span class="linenos" data-linenos="5 "></span></a>. /etc/profile.d/profile-opentsdb.sh
</code></pre></div>
<ul>
<li>8.3 运行配置</li>
</ul>
<p>TODO</p>
<ul>
<li>8.4 服务配置</li>
</ul>
<p>TODO</p>
<h2 id="9">9. 启动集群<a class="headerlink" href="#9" title="Permanent link">&para;</a></h2>
<ul>
<li>首次安装须先格式化</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-34-1" name="__codelineno-34-1"></a><a href="#__codelineno-34-1"><span class="linenos" data-linenos="1 "></span></a><span class="c1"># 首次安装，需在 nn1 节点上执行格式化命令，注: 前提条件是必须先启动 journalnode 服务, 因为格式化需连接到 journal edit 8485 端口写日志)</span>
<a id="__codelineno-34-2" name="__codelineno-34-2"></a><a href="#__codelineno-34-2"><span class="linenos" data-linenos="2 "></span></a>hdfs namenode -format
<a id="__codelineno-34-3" name="__codelineno-34-3"></a><a href="#__codelineno-34-3"><span class="linenos" data-linenos="3 "></span></a>
<a id="__codelineno-34-4" name="__codelineno-34-4"></a><a href="#__codelineno-34-4"><span class="linenos" data-linenos="4 "></span></a><span class="c1"># 如果是 HA 模式则需格式化 zkfc，如果需要手动重置，则需清除 zk 中的数据，通常是: echo &#39;rmr /hadoop-ha&#39;|zkCli.sh</span>
<a id="__codelineno-34-5" name="__codelineno-34-5"></a><a href="#__codelineno-34-5"><span class="linenos" data-linenos="5 "></span></a>hdfs zkfc -formatZK
</code></pre></div>
<ul>
<li>
<p>如果是 HA 模式，则如上格式化操作仅需在 NN1 节点执行即可，注: 在启动 NN2 前，必须先执行 <code>hdfs namenode -bootstrapStandby</code> 来指定
是 standby 节点，否则会认为当前要启动的是 active 节点，导致报错 <code>NameNode not formatted</code>, 待集群就绪后可通过 <code>hdfs haadmin -getServiceState nn1</code> 查看 active 状态.</p>
</li>
<li>
<p><a href="https://gitee.com/wl4g/blogs/blob/master/docs/hadoop/emr-production-deployment/resources/README.md">启动 EMR 集群所有组件</a></p>
</li>
</ul>
<h2 id="10">10. 验证集群<a class="headerlink" href="#10" title="Permanent link">&para;</a></h2>
<ul>
<li>
<p>验证条件 1: 集群所有组件的进程启动后, 检查是否与如上 <a href="#集群部署拓扑">集群部署拓扑</a> 的进程列表一致</p>
</li>
<li>
<p>验证条件 2: 利用 <code>kafka-console-producer.sh</code> 生产数据到 <code>topic1</code>, 能被 spark-streaming 的 SparkSubmit 进程消费并存储进 HBase 的<code>tb_table1</code> 表.</p>
</li>
</ul>
<h2 id="11_1">11. 部署本地开发环境<a class="headerlink" href="#11_1" title="Permanent link">&para;</a></h2>
<blockquote>
<p>部署本地开发环境只需将上述 <code>curl</code> 下配置文件路径中的 <code>xx/emr/xx</code> 改为 <code>xx/emr-local/xx</code> 即可</p>
</blockquote>
<ul>
<li>如下载适用于本地的 hdfs-site.xml</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-35-1" name="__codelineno-35-1"></a><a href="#__codelineno-35-1"><span class="linenos" data-linenos="1 "></span></a>curl -Lk -o <span class="nv">$HADOOP_CONF_DIR</span>/hdfs-site.xml https://gitee.com/wl4g/blogs/raw/master/docs/articles/hadoop/emr-production-deployment/resources/etc/emr-local/hadoop-conf/hdfs-site.xml
</code></pre></div>
<ul>
<li>启动并验证</li>
</ul>
<div class="highlight"><pre><span></span><code><a id="__codelineno-36-1" name="__codelineno-36-1"></a><a href="#__codelineno-36-1"><span class="linenos" data-linenos=" 1 "></span></a><span class="c1"># 首次安装需格式化 hdfs</span>
<a id="__codelineno-36-2" name="__codelineno-36-2"></a><a href="#__codelineno-36-2"><span class="linenos" data-linenos=" 2 "></span></a>hdfs namenode -format
<a id="__codelineno-36-3" name="__codelineno-36-3"></a><a href="#__codelineno-36-3"><span class="linenos" data-linenos=" 3 "></span></a>
<a id="__codelineno-36-4" name="__codelineno-36-4"></a><a href="#__codelineno-36-4"><span class="linenos" data-linenos=" 4 "></span></a><span class="c1"># 以最小的规格启动，只保留必要的服务</span>
<a id="__codelineno-36-5" name="__codelineno-36-5"></a><a href="#__codelineno-36-5"><span class="linenos" data-linenos=" 5 "></span></a>zkServer.sh start
<a id="__codelineno-36-6" name="__codelineno-36-6"></a><a href="#__codelineno-36-6"><span class="linenos" data-linenos=" 6 "></span></a>hadoop-daemon.sh start namenode
<a id="__codelineno-36-7" name="__codelineno-36-7"></a><a href="#__codelineno-36-7"><span class="linenos" data-linenos=" 7 "></span></a>hadoop-daemon.sh start datanode
<a id="__codelineno-36-8" name="__codelineno-36-8"></a><a href="#__codelineno-36-8"><span class="linenos" data-linenos=" 8 "></span></a>hbase-daemon.sh start master
<a id="__codelineno-36-9" name="__codelineno-36-9"></a><a href="#__codelineno-36-9"><span class="linenos" data-linenos=" 9 "></span></a>hbase-daemon.sh start regionserver
<a id="__codelineno-36-10" name="__codelineno-36-10"></a><a href="#__codelineno-36-10"><span class="linenos" data-linenos="10 "></span></a>kafka-server-start.sh -daemon <span class="nv">$KAFKA_CONF_DIR</span>/server.properties
<a id="__codelineno-36-11" name="__codelineno-36-11"></a><a href="#__codelineno-36-11"><span class="linenos" data-linenos="11 "></span></a>
<a id="__codelineno-36-12" name="__codelineno-36-12"></a><a href="#__codelineno-36-12"><span class="linenos" data-linenos="12 "></span></a><span class="c1"># 验证服务</span>
<a id="__codelineno-36-13" name="__codelineno-36-13"></a><a href="#__codelineno-36-13"><span class="linenos" data-linenos="13 "></span></a>hdfs dfs -ls /
<a id="__codelineno-36-14" name="__codelineno-36-14"></a><a href="#__codelineno-36-14"><span class="linenos" data-linenos="14 "></span></a>sqlline.py
<a id="__codelineno-36-15" name="__codelineno-36-15"></a><a href="#__codelineno-36-15"><span class="linenos" data-linenos="15 "></span></a><span class="m">0</span>: jdbc:phoenix:localhost:2181:/hbase&gt; !table
</code></pre></div>

              
            </article>
          </div>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    <script id="__config" type="application/json">{"base": "../../../..", "features": ["search.suggest", "search.highlight", "navigation.tabs", "navigation.expand", "toc.follow", "toc.integrate"], "search": "../../../../assets/javascripts/workers/search.5e67fbfe.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.config.lang": "en", "search.config.pipeline": "trimmer, stopWordFilter", "search.config.separator": "[\\s\\-]+", "search.placeholder": "Search", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version.title": "Select version"}, "version": {"default": "latest", "provider": "mike"}}</script>
    
    
      <script src="../../../../assets/javascripts/bundle.c44cc438.min.js"></script>
      
        <script src="../../../../static/js/util.js"></script>
      
    
  </body>
</html>